{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, display_html\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import scipy as sc\n",
    "from scipy import stats\n",
    "\n",
    "from termcolor import colored\n",
    "\n",
    "# Plotly\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objs as go\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[34m==== Train ====\u001b[0m\n",
      "\u001b[1m\u001b[34mShape: \u001b[0m (3911, 8)\n",
      "\u001b[1m\u001b[34mNaN Values: \u001b[0m 0 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0016926B079C</td>\n",
       "      <td>I think that students would benefit from learn...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0022683E9EA5</td>\n",
       "      <td>When a problem is a change you have to let it ...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00299B378633</td>\n",
       "      <td>Dear, Principal\\n\\nIf u change the school poli...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>003885A45F42</td>\n",
       "      <td>The best time in life is when you become yours...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0049B1DF5CCC</td>\n",
       "      <td>Small act of kindness can impact in other peop...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m\u001b[34m==== Test ====\u001b[0m\n",
      "\u001b[1m\u001b[34mShape: \u001b[0m (3, 2)\n",
      "\u001b[1m\u001b[34mNaN Values: \u001b[0m 0 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>full_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000C359D63E</td>\n",
       "      <td>when a person has no experience on a job their...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000BAD50D026</td>\n",
       "      <td>Do you think students would benefit from being...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00367BB2546B</td>\n",
       "      <td>Thomas Jefferson once states that \"it is wonde...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m\u001b[34m==== Sample Submission ====\u001b[0m\n",
      "\u001b[1m\u001b[34mShape: \u001b[0m (3, 7)\n",
      "\u001b[1m\u001b[34mNaN Values: \u001b[0m 0 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000C359D63E</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000BAD50D026</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00367BB2546B</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# 查看数据\n",
    "def load_data():\n",
    "    train = pd.read_csv( \"F:/桌面存储/feedback-prize-english-language-learning(1)/train.csv\")\n",
    "    test = pd.read_csv(\"F:/桌面存储/feedback-prize-english-language-learning(1)//test.csv\")\n",
    "    sample_submission = pd.read_csv('F:/桌面存储/feedback-prize-english-language-learning(1)//sample_submission.csv')\n",
    "    return train, test, sample_submission\n",
    "\n",
    "def data_info(csv, name=\"Train\"):\n",
    "    print(colored('==== {} ===='.format(name), 'blue', attrs=['bold']))\n",
    "    print(colored('Shape: ', 'blue', attrs=['bold']), csv.shape)\n",
    "    print(colored('NaN Values: ', 'blue', attrs=['bold']), csv.isnull().sum().sum(), '\\n')\n",
    "    #print(colored('Columns: ', 'blue', attrs=['bold']), list(csv.columns))\n",
    "    \n",
    "    display_html(csv.head())\n",
    "    if name != 'Sample Submission': print(\"\\n\")\n",
    "    \n",
    "train, test, sample_submission = load_data()\n",
    "clear_output()  #清除输出\n",
    "\n",
    "names = [\"Train\", \"Test\", \"Sample Submission\"]\n",
    "for i, df in enumerate([train, test, sample_submission]): \n",
    "    data_info(df, names[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text_id         object\n",
       "full_text       object\n",
       "cohesion       float64\n",
       "syntax         float64\n",
       "vocabulary     float64\n",
       "phraseology    float64\n",
       "grammar        float64\n",
       "conventions    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col0,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col3,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col4,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col5,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col6,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col7,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col0,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col1,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col3,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col4,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col5,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col6,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col7,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col0,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col2,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col3,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col5,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col6,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col7,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col0,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col3,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col4,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col5,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col6,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col7,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col0,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col3,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col4,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col5,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col6,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col7,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col0,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col3,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col4,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col5,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col6,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col7{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col1{\n",
       "            background-color:  #75b4d8;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col2{\n",
       "            background-color:  #3383be;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col2{\n",
       "            background-color:  #63a8d3;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col1,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col4,#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col1{\n",
       "            background-color:  #89bedc;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col2{\n",
       "            background-color:  #4292c6;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col1{\n",
       "            background-color:  #f3f8fe;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col1{\n",
       "            background-color:  #c4daee;\n",
       "            color:  #000000;\n",
       "        }#T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col2{\n",
       "            background-color:  #206fb4;\n",
       "            color:  #f1f1f1;\n",
       "        }</style><table id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >count</th>        <th class=\"col_heading level0 col1\" >mean</th>        <th class=\"col_heading level0 col2\" >std</th>        <th class=\"col_heading level0 col3\" >min</th>        <th class=\"col_heading level0 col4\" >25%</th>        <th class=\"col_heading level0 col5\" >50%</th>        <th class=\"col_heading level0 col6\" >75%</th>        <th class=\"col_heading level0 col7\" >max</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327level0_row0\" class=\"row_heading level0 row0\" >cohesion</th>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col0\" class=\"data row0 col0\" >3911.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col1\" class=\"data row0 col1\" >3.127077</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col2\" class=\"data row0 col2\" >0.662542</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col3\" class=\"data row0 col3\" >1.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col4\" class=\"data row0 col4\" >2.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col5\" class=\"data row0 col5\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col6\" class=\"data row0 col6\" >3.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row0_col7\" class=\"data row0 col7\" >5.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327level0_row1\" class=\"row_heading level0 row1\" >syntax</th>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col0\" class=\"data row1 col0\" >3911.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col1\" class=\"data row1 col1\" >3.028254</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col2\" class=\"data row1 col2\" >0.644399</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col3\" class=\"data row1 col3\" >1.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col4\" class=\"data row1 col4\" >2.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col5\" class=\"data row1 col5\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col6\" class=\"data row1 col6\" >3.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row1_col7\" class=\"data row1 col7\" >5.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327level0_row2\" class=\"row_heading level0 row2\" >vocabulary</th>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col0\" class=\"data row2 col0\" >3911.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col1\" class=\"data row2 col1\" >3.235745</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col2\" class=\"data row2 col2\" >0.583148</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col3\" class=\"data row2 col3\" >1.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col4\" class=\"data row2 col4\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col5\" class=\"data row2 col5\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col6\" class=\"data row2 col6\" >3.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row2_col7\" class=\"data row2 col7\" >5.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327level0_row3\" class=\"row_heading level0 row3\" >phraseology</th>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col0\" class=\"data row3 col0\" >3911.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col1\" class=\"data row3 col1\" >3.116850</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col2\" class=\"data row3 col2\" >0.655997</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col4\" class=\"data row3 col4\" >2.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col5\" class=\"data row3 col5\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col6\" class=\"data row3 col6\" >3.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row3_col7\" class=\"data row3 col7\" >5.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327level0_row4\" class=\"row_heading level0 row4\" >grammar</th>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col0\" class=\"data row4 col0\" >3911.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col1\" class=\"data row4 col1\" >3.032856</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col2\" class=\"data row4 col2\" >0.699841</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col3\" class=\"data row4 col3\" >1.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col4\" class=\"data row4 col4\" >2.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col5\" class=\"data row4 col5\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col6\" class=\"data row4 col6\" >3.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row4_col7\" class=\"data row4 col7\" >5.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327level0_row5\" class=\"row_heading level0 row5\" >conventions</th>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col0\" class=\"data row5 col0\" >3911.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col1\" class=\"data row5 col1\" >3.081053</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col2\" class=\"data row5 col2\" >0.671450</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col3\" class=\"data row5 col3\" >1.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col4\" class=\"data row5 col4\" >2.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col5\" class=\"data row5 col5\" >3.000000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col6\" class=\"data row5 col6\" >3.500000</td>\n",
       "                        <td id=\"T_db4368a0_60a2_11ed_9ba6_74d83e0ac327row5_col7\" class=\"data row5 col7\" >5.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1a39428cc10>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 对特征进行归类，筛选出数值型特征：包括int和float数据\n",
    "train.select_dtypes(['int','float']).describe().T.style.background_gradient(cmap='Blues')\n",
    "# 调用describe()函数，输出统计信息，.T对输出数据进行转置操作\n",
    "# 使用 background_gradient() 函数可以对背景颜色进行设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I think that students would benefit from learning at home,because they wont have to change and get up early in the morning to shower and do there hair. taking only classes helps them because at there house they'll be pay more attention. they will be comfortable at home.\\n\\nThe hardest part of school is getting ready. you wake up go brush your teeth and go to your closet and look at your cloths. after you think you picked a outfit u go look in the mirror and youll either not like it or you look and see a stain. Then you'll have to change. with the online classes you can wear anything and stay home and you wont need to stress about what to wear.\\n\\nmost students usually take showers before school. they either take it before they sleep or when they wake up. some students do both to smell good. that causes them do miss the bus and effects on there lesson time cause they come late to school. when u have online classes u wont need to miss lessons cause you can get everything set up and go take a shower and when u get out your ready to go.\\n\\nwhen your home your comfortable and you pay attention. it gives then an advantage to be smarter and even pass there classmates on class work. public schools are difficult even if you try. some teacher dont know how to teach it in then way that students understand it. that causes students to fail and they may repeat the class.              \""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看作文样例\n",
    "train['full_text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I think that students would benefit from learning at home,because they wont have to change and get up early in the morning to shower and do there hair. taking only classes helps them because at there house they'll be pay more attention. they will be comfortable at home.  The hardest part of school is getting ready. you wake up go brush your teeth and go to your closet and look at your cloths. after you think you picked a outfit u go look in the mirror and youll either not like it or you look and see a stain. Then you'll have to change. with the online classes you can wear anything and stay home and you wont need to stress about what to wear.  most students usually take showers before school. they either take it before they sleep or when they wake up. some students do both to smell good. that causes them do miss the bus and effects on there lesson time cause they come late to school. when u have online classes u wont need to miss lessons cause you can get everything set up and go take a shower and when u get out your ready to go.  when your home your comfortable and you pay attention. it gives then an advantage to be smarter and even pass there classmates on class work. public schools are difficult even if you try. some teacher dont know how to teach it in then way that students understand it. that causes students to fail and they may repeat the class.              \""
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 对作文进行正则化操作，清除标点符号\n",
    "import re\n",
    "\n",
    "train['full_text'] = train[\"full_text\"].replace(re.compile(r'[\\n\\r\\t]'), ' ', regex=True)\n",
    "test['full_text'] = test[\"full_text\"].replace(re.compile(r'[\\n\\r\\t]'), ' ', regex=True)\n",
    "train['full_text'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGBOOST Regression with GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_feature_label(data, label):\n",
    "    #data_after = data[(data['price']<400) & (data['price']>1)]\n",
    "    train_features = data[['full_text', 'length', 'paragraphs', 'punctuation_count', 'pc_ratio']]\n",
    "    ### log transform\n",
    "    train_labels =  data[label]\n",
    "    #train_labels[train_labels==0]=0.01\n",
    "    train_labels = np.log(train_labels)\n",
    "    return train_features,train_labels\n",
    "\n",
    "\n",
    "def get_train_test_features(label, df_train, df_test):\n",
    "    nrow_train = df_train.shape[0]\n",
    "    train_features,train_labels = get_feature_label(df_train,label)\n",
    "    df_combine:pd.DataFrame = pd.concat([train_features,df_test[['full_text', 'length', 'paragraphs']]])\n",
    "    #df.concat()这种合并方法，将完整的保留数据（不会丢弃数据，而是根据 index 和 columns 增加行列），并可指定合并的轴。\n",
    "    del df_train\n",
    "    del df_test\n",
    "    del train_features\n",
    "    #删除df_train、df_test、train_features\n",
    "    tfidf = TfidfVectorizer(norm='l2',sublinear_tf=True,ngram_range=(1,3),min_df=10,max_features=500, stop_words = 'english')\n",
    "    X_full_text  = tfidf.fit_transform(df_combine['full_text'])\n",
    "    del(tfidf)\n",
    "    X_full_text = X_full_text[:, np.array(np.clip(X_full_text.getnnz(axis=0) - 1, 0, 1), dtype=bool)]\n",
    "    print ('Dimension of fulltext_features'+str(X_full_text.shape))\n",
    "    gc.collect()\n",
    "    X_numerical = df_combine[['length', 'paragraphs']]\n",
    "    final_features = sparse.hstack((X_full_text, X_numerical)).tocsr()\n",
    "    print ('Dimension of final_features'+ str(final_features.shape))\n",
    "    train_final_features = final_features[:nrow_train]\n",
    "    test_final_features = final_features[nrow_train:]\n",
    "    del final_features\n",
    "    gc.collect()\n",
    "    X = (train_final_features)\n",
    "    y = (train_labels)\n",
    "    return(X,y, test_final_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#选取最好的参数\n",
    "def search_best_params(X,y, label):\n",
    "    regressor=xgb.XGBRegressor(eval_metric='rmsle',\n",
    "                           booster='gbtree',\n",
    "                           objective = 'reg:linear',\n",
    "                           gamma=0,subsample=0.9,\n",
    "                           colsample_bytree=1,\n",
    "                           min_child_weight=1, \n",
    "                           n_jobs=4,\n",
    "                           seed=273\n",
    "                          )\n",
    "    \n",
    "#参数集定义\n",
    "    param_grid = {\"max_depth\":    [4, 5, 7, 9, 11],\n",
    "              \"n_estimators\": [500, 600, 700, 1000, 1500, 2000],\n",
    "              \"learning_rate\": [0.01, 0.015, .1, .25, .5]\n",
    "             }\n",
    "    \n",
    "    param_grid = {\"max_depth\":    [5],\n",
    "              \"n_estimators\": [500],\n",
    "              \"learning_rate\": [0.01]\n",
    "             }\n",
    "    #网格搜索并打印最佳参数\n",
    "    search = GridSearchCV(regressor, param_grid, cv=5).fit(X, y)\n",
    "    print(f\"The best hyperparameters for {label} are \",search.best_params_)\n",
    "    return(search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           text_id                                          full_text  \\\n",
      "0     0016926B079C  I think that students would benefit from learn...   \n",
      "1     0022683E9EA5  When a problem is a change you have to let it ...   \n",
      "2     00299B378633  Dear, PrincipalIf u change the school policy o...   \n",
      "3     003885A45F42  The best time in life is when you become yours...   \n",
      "4     0049B1DF5CCC  Small act of kindness can impact in other peop...   \n",
      "...            ...                                                ...   \n",
      "3906  FFD29828A873  I believe using cellphones in class for educat...   \n",
      "3907  FFD9A83B0849  Working alone, students do not have to argue w...   \n",
      "3908  FFDC4011AC9C  \"A problem is a chance for you to do your best...   \n",
      "3909  FFE16D704B16  Many people disagree with Albert Schweitzer's ...   \n",
      "3910  FFED00D6E0BD  Do you think that failure is the main thing fo...   \n",
      "\n",
      "      cohesion  syntax  vocabulary  phraseology  grammar  conventions  length  \\\n",
      "0          3.5     3.5         3.0          3.0      4.0          3.0    1381   \n",
      "1          2.5     2.5         3.0          2.0      2.0          2.5    2625   \n",
      "2          3.0     3.5         3.0          3.0      3.0          2.5    1661   \n",
      "3          4.5     4.5         4.5          4.5      4.0          5.0    3959   \n",
      "4          2.5     3.0         3.0          3.0      2.5          2.5    1322   \n",
      "...        ...     ...         ...          ...      ...          ...     ...   \n",
      "3906       2.5     3.0         3.0          3.5      2.5          2.5     969   \n",
      "3907       4.0     4.0         4.0          4.0      3.5          3.0    2591   \n",
      "3908       2.5     3.0         3.0          3.0      3.5          3.0    1298   \n",
      "3909       4.0     4.5         4.5          4.0      4.5          4.5    2825   \n",
      "3910       3.5     2.5         3.5          3.0      3.0          3.5    3383   \n",
      "\n",
      "      paragraphs  punctuation_count  pc_ratio  \n",
      "0              1                 21  0.015206  \n",
      "1              1                 21  0.008000  \n",
      "2              1                 36  0.021674  \n",
      "3              1                108  0.027280  \n",
      "4              1                  3  0.002269  \n",
      "...          ...                ...       ...  \n",
      "3906           1                 13  0.013416  \n",
      "3907           1                 37  0.014280  \n",
      "3908           1                 21  0.016179  \n",
      "3909           1                 50  0.017699  \n",
      "3910           1                 46  0.013597  \n",
      "\n",
      "[3911 rows x 12 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0       1381\n",
       "1       2625\n",
       "2       1661\n",
       "3       3959\n",
       "4       1322\n",
       "        ... \n",
       "3906     969\n",
       "3907    2591\n",
       "3908    1298\n",
       "3909    2825\n",
       "3910    3383\n",
       "Name: length, Length: 3911, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "def remove_newline(x):\n",
    "    x = re.sub(r\"\\n\\n\", \"\", x)\n",
    "    x = re.sub(r\"\\r\\n\\r\\n\", \"\", x)\n",
    "    return(x)\n",
    "\n",
    "#以CSV形式导入数据集\n",
    "df_train = pd.read_csv('F:/桌面存储/feedback-prize-english-language-learning(1)/train.csv')\n",
    "#map():将序列中的每一个元素，输入函数，最后将映射后的每个值返回合并，得到一个迭代器。\n",
    "df_train['full_text'] = list(map(remove_newline, df_train['full_text'])) \n",
    "df_train['length'] = list(map(lambda x : len(x), df_train['full_text']))\n",
    "df_train['paragraphs'] = list(map(lambda x : len(x.splitlines()), df_train['full_text']))\n",
    "\n",
    "df_test = pd.read_csv('F:/桌面存储/feedback-prize-english-language-learning(1)/test.csv')\n",
    "df_test['length'] = list(map(lambda x : len(x), df_test['full_text']))\n",
    "df_test['paragraphs'] = list(map(lambda x : len(x.splitlines()), df_test['full_text']))\n",
    "\n",
    "#将完整的df_train['full_text']传入函数：(lambda内，x即为df_train['full_text']\n",
    "#遍历标点符号\n",
    "df_train['punctuation_count'] = df_train['full_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n",
    "df_test['punctuation_count'] = df_test['full_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n",
    "df_train['pc_ratio'] = df_train['punctuation_count']/ df_train['length']\n",
    "df_test['pc_ratio'] = df_test['punctuation_count']/ df_test['length']\n",
    "\n",
    "print(df_train)\n",
    "#df_train['full_text']\n",
    "df_train['length']\n",
    "#df_train['pc_ratio']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_final_model(X, y, search):\n",
    "    regressor=xgb.XGBRegressor(learning_rate = search.best_params_[\"learning_rate\"],\n",
    "                               n_estimators  = search.best_params_[\"n_estimators\"],\n",
    "                               max_depth     = search.best_params_[\"max_depth\"],\n",
    "                               eval_metric='rmsle',\n",
    "                               booster='gbtree',\n",
    "                               objective = 'reg:linear',\n",
    "                               gamma=0,subsample=0.9,\n",
    "                               colsample_bytree=1,\n",
    "                               min_child_weight=1, \n",
    "                               n_jobs=4,\n",
    "                               seed=273\n",
    "                              )\n",
    "#booster='gbtree'：使用哪种助推器。可以是，或者;使用基于树的模型，同时使用线性函数。此处选择树    \n",
    "#X_train,X_test,y_train,y_test = train_test_split(X,y, test_size = 0.2)\n",
    "    regressor.fit(X, y)\n",
    "    return(regressor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of fulltext_features(3914, 500)\n",
      "Dimension of final_features(3914, 502)\n",
      "[11:16:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:16:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:16:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:16:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:17:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:17:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "The best hyperparameters for cohesion are  {'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 500}\n",
      "[11:17:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[3.0, 3.5, 3.5]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import gc\n",
    "from scipy import sparse\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "label = 'cohesion'\n",
    "X,y, test_X = get_train_test_features(label, df_train, df_test); \n",
    "search = search_best_params(X,y, label)\n",
    "xgb_model = get_final_model(X, y, search)\n",
    "\n",
    "#用最佳参数进行预测\n",
    "predictions = xgb_model.predict(test_X)\n",
    "# 即numpy.exp(predictions)计算x的每个元素的e^x\n",
    "cohesion_label = np.exp(predictions)\n",
    "#round()用于数字的四舍五入，round()中默认保留小数位是0\n",
    "cohesion_label = [round(l)/2 for l in cohesion_label*2]\n",
    "\n",
    "#X,y, test_X\n",
    "#search\n",
    "#xgb_model\n",
    "#predictions\n",
    "cohesion_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of fulltext_features(3914, 500)\n",
      "Dimension of final_features(3914, 502)\n",
      "[11:17:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:18:03] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:18:14] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:18:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:18:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:18:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "The best hyperparameters for syntax are  {'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 500}\n",
      "[11:19:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    }
   ],
   "source": [
    "label = 'syntax'\n",
    "X,y, test_X = get_train_test_features(label, df_train, df_test); \n",
    "search = search_best_params(X,y, label)\n",
    "xgb_model = get_final_model(X, y, search)\n",
    "predictions = xgb_model.predict(test_X)\n",
    "syntax_label = np.exp(predictions)\n",
    "syntax_label = [round(l)/2 for l in syntax_label*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of fulltext_features(3914, 500)\n",
      "Dimension of final_features(3914, 502)\n",
      "[11:19:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:19:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:20:00] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:20:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:20:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:20:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "The best hyperparameters for vocabulary are  {'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 500}\n",
      "[11:20:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    }
   ],
   "source": [
    "label = 'vocabulary'\n",
    "X,y, test_X = get_train_test_features(label, df_train, df_test); \n",
    "search = search_best_params(X,y, label)\n",
    "xgb_model = get_final_model(X, y, search)\n",
    "predictions = xgb_model.predict(test_X)\n",
    "vocab_label = np.exp(predictions)\n",
    "vocab_label = [round(l)/2 for l in vocab_label*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of fulltext_features(3914, 500)\n",
      "Dimension of final_features(3914, 502)\n",
      "[11:21:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:21:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:22:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:22:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:22:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:22:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "The best hyperparameters for phraseology are  {'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 500}\n",
      "[11:22:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    }
   ],
   "source": [
    "label = 'phraseology'\n",
    "X,y, test_X = get_train_test_features(label, df_train, df_test); \n",
    "search = search_best_params(X,y, label)\n",
    "xgb_model = get_final_model(X, y, search)\n",
    "predictions = xgb_model.predict(test_X)\n",
    "phrase_label = np.exp(predictions)\n",
    "phrase_label = [round(l)/2 for l in phrase_label*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of fulltext_features(3914, 500)\n",
      "Dimension of final_features(3914, 502)\n",
      "[11:23:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:23:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:23:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:23:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:24:00] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:24:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "The best hyperparameters for grammar are  {'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 500}\n",
      "[11:24:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    }
   ],
   "source": [
    "label = 'grammar'\n",
    "X,y, test_X = get_train_test_features(label, df_train, df_test); \n",
    "search = search_best_params(X,y, label)\n",
    "xgb_model = get_final_model(X, y, search)\n",
    "predictions = xgb_model.predict(test_X)\n",
    "grammar_label = np.exp(predictions)\n",
    "grammar_label = [round(l)/2 for l in grammar_label*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension of fulltext_features(3914, 500)\n",
      "Dimension of final_features(3914, 502)\n",
      "[11:24:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:25:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:25:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:25:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:25:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:25:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "The best hyperparameters for conventions are  {'learning_rate': 0.01, 'max_depth': 5, 'n_estimators': 500}\n",
      "[11:26:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.6.0/src/objective/regression_obj.cu:203: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    }
   ],
   "source": [
    "label = 'conventions'\n",
    "X,y, test_X = get_train_test_features(label, df_train, df_test); \n",
    "search = search_best_params(X,y, label)\n",
    "xgb_model = get_final_model(X, y, search)\n",
    "predictions = xgb_model.predict(test_X)\n",
    "conventions_label = np.exp(predictions)\n",
    "conventions_label = [round(l)/2 for l in conventions_label*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        text_id  cohesion  syntax  vocabulary  phraseology  grammar  \\\n",
      "0  0000C359D63E       3.0     3.0         3.5          3.0      3.0   \n",
      "1  000BAD50D026       3.5     3.0         3.5          3.0      3.0   \n",
      "2  00367BB2546B       3.5     3.5         3.5          3.5      3.5   \n",
      "\n",
      "   conventions  \n",
      "0          2.5  \n",
      "1          3.0  \n",
      "2          3.5  \n"
     ]
    }
   ],
   "source": [
    "df_submit = pd.DataFrame({ 'text_id': df_test['text_id']\n",
    "                           ,'cohesion' : cohesion_label\n",
    "                           , 'syntax' : syntax_label\n",
    "                           , 'vocabulary' : vocab_label\n",
    "                           , 'phraseology' : phrase_label\n",
    "                           , 'grammar' : grammar_label\n",
    "                           , 'conventions' : conventions_label\n",
    "                           })\n",
    "#df_submit.to_csv(\"submission.csv\", index = False)\n",
    "print(df_submit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4f5c1805611cd1695d5dd2f6b80d68eda2615ecd8f121f19c0c7d6be10b13271"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
